/**
 * 
 */

import java.util.ArrayList;
import java.util.PriorityQueue;
import java.io.*;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * @author diana and lpark
 *
 */
public class Crawler {
	public PriorityQueue<String> linkQueue = new PriorityQueue<String>();
	public ArrayList<String> beenTo = new ArrayList<String>();
	public String seedurl=null;
	
	public void crawl (String url) {
		seedurl = url;
		retrieveLinksAndData(url);
		while (linkQueue.size() > 0){
			String nextlink = linkQueue.poll();
			retrieveLinksAndData(nextlink);
		}
	}
	public void retrieveLinksAndData (String url) {
		beenTo.add(url);
		String newlink = null;
		Document doc;
		try {

			// need http protocol
			doc = Jsoup.connect(url).get();
			
			//Send text to indexer
			String contents = doc.body().text();
			Indexer indexer = new Indexer();
			indexer.add(url, contents);
	 
			// get all links
			Elements links = doc.select("a[href]");
			
			for (Element link : links) {
				// get the value from href attribute

				newlink = link.attr("abs:href");
				if (newlink.length() > 0){

					if (newlink.contains(seedurl)){
							checkAndAdd(newlink);
					}					
				}
				
			}
	 
		} catch (IOException e) {
			e.printStackTrace();
		}	
	}
	private void checkAndAdd(String newlink) {
		if (! beenTo.contains(newlink) && ! linkQueue.contains(newlink)){
			linkQueue.add(newlink);
		}	
	}
}